#!/usr/bin/perl -w
#

use LWP::UserAgent;
use XML::Simple;

my @args=@ARGV;
my %options=(
    "-verbose"=>"false",
);
my %urls=(
    "home_page"=>"http://www.qiushibaike.com",
    "latest"=>"http://www.qiushibaike.com/groups/2/latest",
);
my $db_name="database.xml";

&main();

# Entry point
sub main{
    my %db=();
    if( -e $db_name){
        my $tdb=XMLin($db_name);
        %db=%$tdb;
        defined($db{'config'}->{'last_url'}) && ($urls{'next_page'}=$db{'config'}->{'last_url'});
        defined( $db{'config'}->{'image_dir'}) || ($db{'config'}->{'image_dir'}="images");
    }else{
        $db{'config'}->{'max_id'}=0;
        $db{'config'}->{'qiushi_count'}=0;
        $db{'config'}->{'last_url'}=$urls{'latest'};
        $db{'config'}->{'image_url'}="images";
        $urls{'next_page'}=$urls{'latest'};
    }
    $urls{'image_dir'}=$db{'config'}->{'image_dir'};

    my $op_count=0;
    while( $urls{'next_page'} ne 'none'){# and $temp_limit<5){
        $op_count+=1;
        my $filename=retrieve_page($urls{"next_page"});
        my %articles=parse_articles($filename);
        foreach my $id (keys(%articles)){
            # If target Qiushi is already there, skip it.
            if( !defined($db{'qiushi'}->{"qs$id"})){
                #print "Qiushi id: $_\n$articles{$_}->{'content'}\n";
                print "Qiushi id: $id\n";
                $db{'qiushi'}->{"qs$id"}->{'content'}=$articles{$id}->{'content'};
                $db{'qiushi'}->{"qs$id"}->{'date'}=$articles{$id}->{'post-date'};
                $db{'qiushi'}->{"qs$id"}->{'tags'}=$articles{$id}->{'tags'};
                if( $id > $db{'config'}->{'max_id'}){
                    $db{'config'}->{'max_id'}=$id;
                }
                $db{'config'}->{'qiushi_count'} = $db{'config'}->{'qiushi_count'}+1;
            }
        }
        print "Next page is: $urls{'next_page'}\n";

        if($op_count % 10 ==0 ){
            print "Writing back to file...\n";
            $db{'config'}->{'last_url'}=$urls{'next_page'};
            write_database(\%db);
            #return;
        }
    }
}

# Write back to db
sub write_database{
    my ($content_ref)=@_;
    if( defined( $content_ref)){
        my $xmlout=XMLout($content_ref, 'RootName'=>"QiuShiBaiKe_Offline_Database");
        open XMLDB,"+>$db_name";
        if( defined(XMLDB)){
            print XMLDB $xmlout;
            close XMLDB;
        }
    }
}

# Retrieve one single resource from remote server
sub retrieve_resource{
    my ($url, $to)=@_;
    (defined ($url) && defined($to)) || return $url;
    print "Retrieving URL $url\n";
    my $file_cache=$to;
    my $user_agent=LWP::UserAgent->new(
        "agent"=>"Mozilla/5.0",
        "from"=>'robot@robot.com'
    );
    $user_agent->get( $url, ':content_file'=> "$file_cache");
    return $file_cache;
}

# Retrieve one page from remote server
sub retrieve_page{
    my ($url)=@_;
    return &retrieve_resource($url, "cache.txt");
}

# Retrieve one image from remote server
sub retrieve_image{
    my ($rel)=@_;
    defined( $rel) || die "Undefined URL in retrieve_image()";
    my $full_url="$urls{'home_page'}$rel";
    my $loc_url=$rel;
    $loc_url=~s/[\/\\]/_/g;
    return &retrieve_resource( $full_url, "$image_dir{'image_dir'}/$loc_url");
}

# Parse all the articles out from one html file;
sub parse_articles{
    my ($filename)=@_;
    defined( $filename) || die "Bad arguements to parse_articles()";
    #my %articles=%$db;
    my %result_set=();
    my %tag_watermarks=(
        "<p\\s+class=\\\"\\S+\\\"\\s+style=\\\"\\S+\\\">"=>"",
        "<a\\s+href=\\\"\\S+\\\"\\s+rel=\\\"\\S+\\\">"=>"",
        "<\\/p>"=>"",
        "<\\/a>"=>";",
    );
    my %content_watermarks=(
        "\n"=>"",
        "<\\/p>"=>"\n",
        "<\\S+>"=>"",
        "\\&nbsp\\;"=>" ",
        "<\\S+\\s+class=\\\'watermark\\\'>[\\[\\(].+[\\]\\)]"=>"",
    );

    open FILE, "<$filename";
    if( defined FILE){
        my $article="";
        my $article_id=0;
        my ($year,$month,$day,$hour,$min);
        my $article_start=0;
        my @tags={};
        foreach my $line(<FILE>){
            if( $line =~/<div\s+class=\"qiushi_body\s+article\s+\"\s+id="article(\d+)\"\s+title=\"(\d+).(\d+).(\d+)\s+(\d+):(\d+)/ ){
                $article_start=1;
                $article_id=$1;
                $year=$2;
                $month=$3;
                $day=$4;
                $hour=$5;
                $min=$6;
                next;
            }

            if( $line =~/<p\s+style="float:right">/){
                #Got one final article;
                my $post_date="$year-$month-$day-$hour";
                #print "article $article_id posted at $year-$month-$day-$hour:$min\n$article\n";
                $result_set{$article_id}->{'content'}=$article;
                $result_set{$article_id}->{'post-date'}=$post_date;
                $article="";
                #$article_id=0;
                $article_start=0;
                next;
            }
            if( $article_start==1){
                #$line=~/^\s+$/ && next;
                $line=~s/\n//g;
                $line=~s/<\/p>/\n/g;
                $line=~s/<\S+>//g;
                $line=~s/\&nbsp\;/ /g;
                $line=~s/<\S+\s+class=\'watermark\'>[\[\(].+[\]\)]//g;
                #foreach my $wmark( keys( %content_watermarks)){
                #    $line=~s/$wmark/$content_watermarks{$wmark}/g;
                #}
                $article="$article $line";
                next;
            }

            #tag started;
            if( $line=~/<p\s+class=\"tags\".+>/){
                foreach my $wmark ( keys( %tag_watermarks)){
                    $line=~s/$wmark/$tag_watermarks{$wmark}/g;
                }
                #print "Tags: $line\n";
                $result_set{$article_id}->{'tags'}=$line;
                $article_id=0;
                next;
            }

            # Parse the URL to latest Qiushi page;
            if( $line=~/<a\s+href=\"(\S+)\">最新<\/a>/){
                ($urls{'latest'} ne "$urls{'home_page'}$1") && ($urls{'latest'}="$urls{'home_page'}");
                next;
            }
            # Parse the URL of next page;
            if( $line=~/<a\s+href=\"(\S+)\"\s+class=\"next_page\"\s+rel="next">下一页/){
                #$result_set{header'}'next_page'}="$urls{'home_page'}$1";
                $urls{'next_page'}="$urls{'home_page'}$1";
                next;
            }
            # Determine if this is the last page;
            if( $line=~/<span\s+class=\"disabled\s+next_page\">下一页/){
                #$result_set{'last_page'}='true';
                $urls{'next_page'}='none';
                next;
            }
        }
        close FILE;
        return %result_set;
    }
}

# Parse the URL of latest qiushi from HTML docs;
sub parse_latest_url{
    my ($filename)=@_;
    my @tmp=$filename;

    foreach my $line (@tmp){
        if( $line =~/<a\s+href=\"(\S+)\">最新<\/a>/){
            $urls{'latest'}="$urls{'home_page'}$1";
        }
    }
}

# Display usage message
sub show_usage{

}
